from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/probabilityTableUsingProfIdea20130413"
outputFileHandler = open(outputFileName,"w")

molecularCellDict = {}
denominatorCellDict = {}


print "Step1: Do table:Molecular begins..."
# CURRENT version of generating the 3 tables
# Let's generate the table:molecular first
classLabelList = ["ROW1","ROW2","ROW3","ROW4","ROW5"]

molecularDict = {}
# should have number of sum values for the above 4 rows
valueForTheFinalRowDictForMolecularTable = {}
inputGuideFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/rangesForEachCellWithEvenValueFor100KQueries_1_10%"
inputGuideFileHandler = open(inputGuideFileName,"r")
dataLine = inputGuideFileHandler.readline()
while dataLine:
    headInfoLine = dataLine
    headInfoLineElements = headInfoLine.strip().split(" ")
    freq = int(headInfoLineElements[0])
    totalNumOfQueryTerms = int(headInfoLineElements[1])
    # ignore the number of ranges: 5
    # averageGap = int( headInfoLineElements[3] )

    if freq not in molecularDict:
        molecularDict[freq] = []
            
    for i in range(0,5):
        # do 5 times
        dataInfoLine = inputGuideFileHandler.readline()
        dataInfoLineElements = dataInfoLine.strip().split(" ")
        (beginningRangeID,endingRangeID,totalFreqForThisBigRange) = (int(dataInfoLineElements[0]),int(dataInfoLineElements[1]),int(dataInfoLineElements[2]) )
        molecularDict[freq].append( (beginningRangeID,endingRangeID,totalFreqForThisBigRange) )
    
    # ignore the empty line
    dataLine = inputGuideFileHandler.readline()
    # get the header line
    dataLine = inputGuideFileHandler.readline()

print "len(molecularDict):",len(molecularDict)
for i in range(0,20):
    print i,molecularDict[i]

inputGuideFileHandler.close()

outputFileHandler.write("table:numerator:freqOfFreqForTheLexiconTerm" + "\n")
outputFileHandler.write("NUM_OF_QUERY_TERMS_NEW:2155" + "\n")
outputFileHandler.write("NUM_OF_QUERIS_COUNT:FROM_85K_to_95K" + "\n")
outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
outputFileHandler.write("********************" + "\n")
for index,classLabel in enumerate(classLabelList):
    outputLine = classLabel + " "
    for i in range(0,20):
        (beginningRangeID,endingRangeID,totalFreqForThisBigRange) = molecularDict[i][index]
        cellKey = classLabel + "_" + str(i)
        if cellKey not in molecularCellDict:
            molecularCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(totalFreqForThisBigRange)
        outputLine += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(totalFreqForThisBigRange) + " "
        if i not in valueForTheFinalRowDictForMolecularTable:
            valueForTheFinalRowDictForMolecularTable[i] = totalFreqForThisBigRange
        else:
            valueForTheFinalRowDictForMolecularTable[i] += totalFreqForThisBigRange
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine)

# add the row for the sum of the above 5 rows
finalOutputLine = "SUM" + " "
# This logic can be attained from another part of the program logic
for i in range(0,20):
    # put the values of the last row into the molecularCellDict as well.
    cellKey = "SUM" + "_" + str(i)
    if cellKey not in molecularCellDict:
        molecularCellDict[cellKey] = "[0,999]:" + str(valueForTheFinalRowDictForMolecularTable[i])
    finalOutputLine += "[0,999]:" + str( valueForTheFinalRowDictForMolecularTable[i] ) + " "

finalOutputLine = finalOutputLine.strip() + "\n"
outputFileHandler.write(finalOutputLine)
outputFileHandler.write("\n")



print "Step2: Do table:denominator begins..."
outputFileHandler.write("table:denominator:freqOfFreqForTheLexiconTerm" + "\n")
outputFileHandler.write("NUM_OF_QUERY_TERMS_SEEN_WHICH_FREQ_LESS_THAN_20:32752" + "\n")
outputFileHandler.write("NUM_OF_QUERIS_COUNT:Head_85K" + "\n")
outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
outputFileHandler.write("********************" + "\n")

valueForTheFinalRowDictForDenominatorTable = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_2D_without_query_terms_fix_zerop_freq_sortedByMyOwnWay"
inputFileHandler = open(inputFileName,"r")

# Current version
smallRangeCellValue = -1
currentFreqAccumulateCounter = 0
smallRangeCellKeyFreq = -1
smallRangeCellKeyRangeID = -1

outputRow1 = "ROW1" + " "
outputRow2 = "ROW2" + " "
outputRow3 = "ROW3" + " "
outputRow4 = "ROW4" + " "
outputRow5 = "ROW5" + " "
finalOutputRow = "SUM" + " "

for i in range(0,20):
    for index,tuple in enumerate( molecularDict[i] ):
        (beginningRangeID,endingRangeID,_) = tuple
        
        dataLine = inputFileHandler.readline()
        dataLineElements = dataLine.strip().split(" ")
        smallRangeCellKey = dataLineElements[0]
        smallRangeCellValue = int(dataLineElements[1])
        smallRangeCellKeyElements = smallRangeCellKey.split("_")
        smallRangeCellKeyFreq = int(smallRangeCellKeyElements[0])
        smallRangeCellKeyRangeID = int(smallRangeCellKeyElements[1])
        
        while smallRangeCellKeyFreq == i and smallRangeCellKeyRangeID >= beginningRangeID and smallRangeCellKeyRangeID <= endingRangeID:
            # for debug ONLY
            # print "smallRangeCellValue:",smallRangeCellValue,"smallRangeCellKeyFreq:",smallRangeCellKeyFreq,"smallRangeCellKeyRangeID:",smallRangeCellKeyRangeID
            currentFreqAccumulateCounter += smallRangeCellValue
            dataLine = inputFileHandler.readline()
            if dataLine.strip() != "":
                dataLineElements = dataLine.strip().split(" ")
                smallRangeCellKey = dataLineElements[0]
                smallRangeCellValue = int(dataLineElements[1])
                smallRangeCellKeyElements = smallRangeCellKey.split("_")
                smallRangeCellKeyFreq = int(smallRangeCellKeyElements[0])
                smallRangeCellKeyRangeID = int(smallRangeCellKeyElements[1])
            else:
                # This is also the line index
                if index == 0:
                    cellKey = "ROW1" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        # pass
                        denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
                    outputRow1 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
                elif index == 1:
                    cellKey = "ROW2" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        # pass
                        denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
                    outputRow2 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
                elif index == 2:
                    cellKey = "ROW3" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        # pass
                        denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
                    outputRow3 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
                elif index == 3:
                    cellKey = "ROW4" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        # pass
                        denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
                    outputRow4 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
                elif index == 4:
                    cellKey = "ROW5" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        # pass
                        denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
                    outputRow5 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
                
                if i not in valueForTheFinalRowDictForDenominatorTable:
                    valueForTheFinalRowDictForDenominatorTable[i] = currentFreqAccumulateCounter
                else:
                    valueForTheFinalRowDictForDenominatorTable[i] += currentFreqAccumulateCounter
                
                # print "PASS","Value:",currentFreqAccumulateCounter
                
                
                # print "valueForTheFinalRowDictForDenominatorTable:",valueForTheFinalRowDictForDenominatorTable
                
                # constructing the finalRow
                # This logic can be attained from another part of the program logic
                finalOutputLine = "SUM" + " "
                for i in range(0,20):
                    # put the values of the last row into the molecularCellDict as well.
                    cellKey = "SUM" + "_" + str(i)
                    if cellKey not in denominatorCellDict:
                        denominatorCellDict[cellKey] = "[0,999]:" + str(valueForTheFinalRowDictForDenominatorTable[i])
                    
                    finalOutputLine += "[0,999]:" + str( valueForTheFinalRowDictForDenominatorTable[i] ) + " "
                
                finalOutputLine = finalOutputLine.strip() + "\n"
                
                # output check for the following lines:
                # print "outputRow1:",outputRow1
                # print "outputRow2:",outputRow2
                # print "outputRow3:",outputRow3
                # print "outputRow4:",outputRow4
                # print "outputRow5:",outputRow5
                # print "finalOutputLine:",finalOutputLine
                
                outputFileHandler.write(outputRow1 + "\n")
                outputFileHandler.write(outputRow2 + "\n")
                outputFileHandler.write(outputRow3 + "\n")
                outputFileHandler.write(outputRow4 + "\n")
                outputFileHandler.write(outputRow5 + "\n")
                outputFileHandler.write(finalOutputLine + "\n")
                outputFileHandler.write("\n")
                

                outputFileHandler.write("\n")
                outputFileHandler.write("table:probability" + "\n")
                outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
                outputFileHandler.write("********************" + "\n")
                # Current Version
                
                print "len(denominatorCellDict):",len(denominatorCellDict)
                print "len(molecularCellDict):",len(molecularCellDict)
                
                
                # need to be debugged
                for classLabel in classLabelList:
                    outputLine = classLabel + " "
                    for i in range(0,20):
                        key = classLabel + "_" + str(i) 
                        # This judgement statement still needed to be judged
                        if float(molecularCellDict[key].split(":")[1]) == 0:
                            outputLine += molecularCellDict[key].split(":")[0] + ":" + str( float(molecularCellDict[key].split(":")[1]) ) + " "
                        else:
                            print "key:",key
                            print "molecularCellDict[key]:",molecularCellDict[key]
                            print "denominatorCellDict[key]:",denominatorCellDict[key]
                            print "molecularCellDict:",molecularCellDict
                            print "denominatorCellDict:",denominatorCellDict
                            outputLine += molecularCellDict[key].split(":")[0] + ":" + str( float( molecularCellDict[key].split(":")[1] ) / float( denominatorCellDict[key].split(":")[1] ) ) + " "
                        
                    outputLine = outputLine.strip() + "\n"
                    outputFileHandler.write(outputLine)
                
                # constructing the final output line for the probability table
                finalOutputLine = "SUM_PROBABILITY" + " "
                for i in range(0,20):
                    cellKey = "SUM" + "_" + str(i)
                    finalOutputLine += molecularCellDict[cellKey].split(":")[0] + ":" + str( float( molecularCellDict[cellKey].split(":")[1] ) / float( denominatorCellDict[cellKey].split(":")[1] ) ) + " "
                finalOutputLine = finalOutputLine.strip() + "\n"
                outputFileHandler.write(finalOutputLine)
                
                outputFileHandler.close()
                print "EOF and end PROCESSING"
                
                exit(0)

        # print "PASS","Value:",currentFreqAccumulateCounter
        
        if index == 0:
            cellKey = "ROW1" + "_" + str(i)
            if cellKey not in denominatorCellDict:
                denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
            outputRow1 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
        elif index == 1:
            cellKey = "ROW2" + "_" + str(i)
            if cellKey not in denominatorCellDict:
                denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
            outputRow2 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
        elif index == 2:
            cellKey = "ROW3" + "_" + str(i)
            if cellKey not in denominatorCellDict:
                denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
            outputRow3 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
        elif index == 3:
            cellKey = "ROW4" + "_" + str(i)
            if cellKey not in denominatorCellDict:
                denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
            outputRow4 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "
        elif index == 4:
            cellKey = "ROW5" + "_" + str(i)
            if cellKey not in denominatorCellDict:
                denominatorCellDict[cellKey] = "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter)
            outputRow5 += "[" + str(beginningRangeID) + "," + str(endingRangeID) + "]" + ":" + str(currentFreqAccumulateCounter) + " "

        if i not in valueForTheFinalRowDictForDenominatorTable:
            valueForTheFinalRowDictForDenominatorTable[i] = currentFreqAccumulateCounter
        else:
            valueForTheFinalRowDictForDenominatorTable[i] += currentFreqAccumulateCounter
                            
        # reset to be 0
        currentFreqAccumulateCounter = 0
        # add the missing smallRangeCellValue
        currentFreqAccumulateCounter += smallRangeCellValue
        # for debug ONLY
        # print "smallRangeCellValue:",smallRangeCellValue,"smallRangeCellKeyFreq:",smallRangeCellKeyFreq,"smallRangeCellKeyRangeID:",smallRangeCellKeyRangeID 

outputFileHandler.close()

'''
# OLD version which used to generate the two old tables.
# make the easy looking table for prof to check
# also with some numbers in the denominator smoothing(average the neighbours)
classLabelList = ["VF","F","M","NF","VR"]
denominatorDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/freqOfFreqInQueries_head_85K_0_85%_2D_without_query_terms_sortedByMyOwnWay"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    key = lineElements[0]
    value = int(lineElements[1])
    if key not in denominatorDict:
        denominatorDict[key] = value
    else:
        print "mark1"
        exit(1)

print "len(denominatorDict):",len(denominatorDict)
# do a smoothing for the value of 0
# the smoothing is very simple, just average the two sides.

#print "smoothing in..."
#for classLabel in classLabelList:
#    outputLine = classLabel + " "
#    for i in range(0,20):
#        key = str(i) + "_" + classLabel
#        if denominatorDict[key] == 0:
#            print "original:",key,denominatorDict[key]
#            previousAuxSmoothingKey = str(i-1) + "_" + classLabel
#            nextAuxSmoothingKey = str(i+1) + "_" + classLabel
#            denominatorDict[key] = (denominatorDict[previousAuxSmoothingKey] + denominatorDict[nextAuxSmoothingKey])/2
#            print "     new:",key,denominatorDict[key]
#print "smoothing out."        
        
# this is the checking of the effectiveness of the smoothing method
#for key in denominatorDict:
#    if denominatorDict[key] == 0:
#        print key,denominatorDict[key]
#        print "not expected"
#        exit(1)


inputFileHandler.close()

molecularDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/100KQueries_1_10%_without_query_terms_sortedByMyOwnWay"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    key = lineElements[0]
    value = int(lineElements[1])
    if key not in molecularDict:
        molecularDict[key] = value
    else:
        print "mark2"
        exit(1)

print "len(molecularDict):",len(molecularDict)
inputFileHandler.close()

# One output file containing 3 tables
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/probabilityTableUsingProfIdea20130410"
outputFileHandler = open(outputFileName,"w")

outputFileHandler.write("RangesSettingForVR,NF,M,F,VF" + "\n")
outputFileHandler.write("VR:[1-100):36875967" + "\n")
outputFileHandler.write("NF:[100-5000):807526" + "\n")
outputFileHandler.write("M:[5000-80,000):36148" + "\n")
outputFileHandler.write("F:[80,000-600,000):6837" + "\n")
outputFileHandler.write("VF:[600,000-ending):2141" + "\n")

outputFileHandler.write("\n")
outputFileHandler.write("table:denominator:freqOfFreqForTheLexiconTerm" + "\n")
outputFileHandler.write("NUM_OF_QUERY_TERMS_SEEN_WHICH_FREQ_LESS_THAN_20:32752" + "\n")
outputFileHandler.write("NUM_OF_QUERIS_COUNT:Head_85K" + "\n")
outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
outputFileHandler.write("********************" + "\n")
for classLabel in classLabelList:
    outputLine = classLabel + " "
    for i in range(0,20):
        key = str(i) + "_" + classLabel
        outputLine += str(denominatorDict[key]) + " "
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine)

outputFileHandler.write("\n")
outputFileHandler.write("table:molecular:freqOfFreqForTheLexiconTerm" + "\n")
outputFileHandler.write("NUM_OF_QUERY_TERMS_NEW:2155" + "\n")
outputFileHandler.write("NUM_OF_QUERIS_COUNT:FROM_85K_to_95K" + "\n")
outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
outputFileHandler.write("********************" + "\n")
for classLabel in classLabelList:
    outputLine = classLabel + " "
    for i in range(0,20):
        key = str(i) + "_" + classLabel
        outputLine += str(molecularDict[key]) + " "
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine)


outputFileHandler.write("\n")
outputFileHandler.write("table:probability" + "\n")
outputFileHandler.write("Freq 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19" + "\n")
outputFileHandler.write("********************" + "\n")
for classLabel in classLabelList:
    outputLine = classLabel + " "
    for i in range(0,20):
        key = str(i) + "_" + classLabel
        
        if molecularDict[key] == 0:
            outputLine += str(molecularDict[key]) + " "
        else:
            outputLine += str(molecularDict[key]/denominatorDict[key]) + " "
        
    outputLine = outputLine.strip() + "\n"
    outputFileHandler.write(outputLine) 
outputFileHandler.close()
'''

